#!/usr/bin/env bash

numnodes=3072
numcores=24
fftsize=3072
npyfactor=1

for nppnodes in 24 4 2 1;
do
  let nppwidth=$numnodes*$numcores
  let nthreads=$numcores/$nppnodes
  let np=$nppwidth/$nthreads
  let npy=$nppnodes*$npyfactor

  title=xe6.a2a.s${fftsize}.p${numnodes}x${nppnodes}.c${nppwidth}

  cat << END > $title.pbs
##PBS -q debug
#PBS -q regular
#PBS -A mp160
#PBS -l mppwidth=$nppwidth
#PBS -l walltime=00:10:00
#PBS -N xe6.a2a
#PBS -e $title.\$PBS_JOBID.err
#PBS -o $title.\$PBS_JOBID.out
#PBS -V
 
cd \$PBS_O_WORKDIR

aprun -n $np -N $nppnodes -d $nthreads /scratch/scratchdirs/jkim/hpccodelets/fftw-module/bin/fft3d_comm -s $fftsize -y $npy

END

  cat $title.pbs
qsub $title.pbs

done
